# -*- coding: utf-8 -*-
import scrapy, re
from fang.items import NewHouseItem, EsfItem
from scrapy_redis.spiders import RedisSpider

class FtxSpiderSpider(RedisSpider):
    name = 'ftx_spider'
    allowed_domains = ['fang.com']
    # start_urls = ['http://www.fang.com/SoufunFamily.htm']
    redis_key = "ftx:start_urls"

    def parse(self, response):
        trs = response.xpath('//div[@class="outCont"]//tr')
        province = None
        for tr in trs:
            tds = tr.xpath('.//td[not(@class)]')
            province_td = tds[0]
            province_text = province_td.xpath('.//text()').get()
            province_text = re.sub(r'\s', '', province_text)
            if province_text:
                province = province_text
            if province_text == '其它':
                continue
            city_td = tds[1]
            city_links = city_td.xpath('./a')
            for city_link in city_links:
                city = city_link.xpath('./text()').get()
                city_url = city_link.xpath('./@href').get()
                scheme = city_url.split('//')[0]
                domain = city_url.split('//')[1]
                if '北京' in city:
                    newhouse_url = 'http://newhouse.fang.com/house/s/'
                    esf_url = 'http://esf.fang.com/'
                else:
                    newhouse_url = scheme + '//' + 'newhouse.' + domain + 'house/s/'
                    esf_url = scheme + '//' + 'esf.' + domain
                # print(province,city)
                yield scrapy.Request(url=newhouse_url, callback=self.parse_newhouse, meta={'info': (province, city)})
                yield scrapy.Request(url=esf_url, callback=self.parse_esf, meta={'info': (province, city)})

                # 这两个break限制了爬虫只爬取一个城市。
                break
            break

    def parse_newhouse(self, response):
        province, city = response.meta.get('info')
        item = NewHouseItem(province=province, city=city)
        lis = response.xpath('//div[@id="newhouse_loupai_list"]/ul/li')
        if not lis:
            return
        for li in lis:
            name = li.xpath('.//div[@class="nlcd_name"]/a/text()').get()
            # 如果没有小区名字，证明这个li标签中没有符合的房子。
            if name is not None:
                name = name.strip()
            else:
                continue
            item['name'] = name
            house_type = li.xpath('.//div[contains(@class,"house_type")]/a/text()').getall()
            item['types'] = house_type
            price = ''.join(li.xpath('.//div[@class="nhouse_price"]//text()').getall())
            price = re.sub(r'\s', '', price)
            item['price'] = price
            area = ''.join(li.xpath('.//div[contains(@class,"house_type")]/text()').getall())
            area = re.sub(r'\s|/|－', '', area)
            item['area'] = area
            address = li.xpath('.//div[@class="address"]/a/@title').get()
            item['address'] = address
            region = li.xpath('.//div[@class="address"]/a/span/text()').get()
            if region:
                region = re.sub(r'\s|\[|\]', '', region)
            else:
                region = None
            item['region'] = region
            sale = li.xpath('.//span[@class="inSale" or @class="forSale" or @class="outSale"]/text()').get()
            item['sale'] = sale
            origin_url = li.xpath('.//div[@class="nlcd_name"]/a/@href').get()
            item['origin_url'] = origin_url
            # print(item)
            # print('*'*30)

            yield item

        # 下一页的按钮
        next_url = response.xpath('//div[@class="page"]//a[@class="next" and position()>5]/@href').get()
        if next_url:
            url = response.urljoin(next_url)
            # print('下一页链接：',url)
            yield scrapy.Request(url=response.urljoin(next_url), callback=self.parse_newhouse,
                                 meta={'info': (province, city)})
        # 如果没有下一页的按钮，则自行定义链接。
        else:
            current_url = response.url
            # print(current_url)
            page = re.search(r'.+fang.com/house/s/b9(\d{0,2})', current_url).group(1)
            next = int(page) + 1
            next_url = re.sub(r'b9\d.*', 'b9' + str(next) + '/', current_url)
            yield scrapy.Request(url=next_url, callback=self.parse_newhouse,
                                 meta={'info': (province, city)})

    def parse_esf(self, response):
        province, city = response.meta.get('info')
        item = EsfItem(province=province, city=city)
        current_page = response.xpath('//div[@id="list_D10_15"]/span[@class="on"]')
        if not current_page:
            return
        dls = response.xpath('//div[contains(@class,"shop_list_4")]/dl')
        for dl in dls:
            title = dl.xpath('.//h4/a/@title').get()
            if title is not None:
                title = title.strip()
            else:
                continue
            item['title'] = title
            name = dl.xpath('.//p[@class="add_shop"]/a/text()').get().strip()
            item['name'] = name
            price = ''.join(dl.xpath('.//span[@class="red"]//text()').getall())
            item['price'] = price
            ave_price = dl.xpath('.//dd[@class="price_right"]/span[2]/text()').get()
            item['ave_price'] = ave_price
            infos = ''.join(dl.xpath('.//p[@class="tel_shop"]//text()').getall())
            infos = re.sub(r'\s', '', infos)
            for list in infos.split('|'):
                if '厅' in list:
                    item['types'] = list
                if '㎡' in list:
                    item['area'] = list
                if '层' in list:
                    item['floor'] = list
                if '向' in list:
                    item['orientation'] = list
                if '建' in list:
                    item['build_year'] = list
            origin_url = dl.xpath('.//h4/a/@href').get()
            item['origin_url'] = response.urljoin(origin_url)

            # print(item)
            # print('*'*30)
            yield item

        current_url = response.url
        current_page = current_page.xpath('./text()').get().strip()
        total_page_text = response.xpath('//div[@id="list_D10_15"]/p[last()]/text()').get().strip()
        total_page = re.search(r'共(.*)页', total_page_text).group(1)
        if int(current_page) > int(total_page):
            return
        else:
            next = int(current_page) + 1
            # print(next)
            next_url = re.sub(r'com/.*', 'com/house/i3' + str(next) + '/', current_url)
            yield scrapy.Request(url=next_url, callback=self.parse_esf,
                                 meta={'info': (province, city)})
